#抓取豆瓣电影TOP100输出为html格式
import requests
import bs4
import re
from bs4 import BeautifulSoup


def getHTMLText(url):
    try:
        r = requests.get(url,timeout = 30)
        r.raise_for_status()
        r.encoding = "utf-8"#r.apparent_encoding
        return r.text
    except:
        return ""
    
#数据处理
def fillVidoList(ulist,html):
    try:
        soup = BeautifulSoup(html,"html.parser")
        #正则得到电影名称:{空格}肖申克的救赎\n/{空格}<span style="font-size:13px;">'        
        temp = re.findall(r'.*?\n.*\/\s\<span style=\"font-size:13px;\">',html)
        for i in range(len(temp)):
            Str = re.findall(r'[\u4e00-\u9fa5]+',temp[i].split("\n")[0])[0] #去掉空格 #剪切出电影名
            alias =  soup.select(".pl2 > a > span ")[i].string              #过滤得到电影别名
            Vurl = soup.select(".pl2 > a ")[i].attrs["href"]                #得到电影链接
            ulist.append([Str,alias,Vurl])
        #print(ulist)
    except:
        print("解析出错！")
        
        
#打印到屏幕        
def output(ulist,num):
    tplt = "{0:^10}\t{1:{4}^10}\t{2:{4}<20}\t{3:<50}"
    print(tplt.format("排名","电影名","电影别名","观看链接",chr(12288)))
    print(tplt.format("-----","-----","-------","-------","-"))
    for i in range(num):
        u = ulist[i]
        print(tplt.format(i+1,u[0],u[1],u[2],chr(12288)))
    print(tplt.format("-----","-----","-------","-------","-"))
    
#写入文件...默认为html格式Flag=True,自定义格式Flag=False
def wirterFile(ulist,filename,Flag):
#try:        
    if Flag:
        fileOutHtml(ulist,filename)
            
    else:
        fileOut(ulist,filename)
#except:
    #print("写入文件出错！")

def fileOutHtml(ulist,filename):
    temp = filename.split(".")[1]#检查是否是html格式如果不是则Flag=False
    if(temp != "html"):
        wirterFile(ulist,filename,False)
    else:
        #美化链接
        link = "<link rel=\"stylesheet\" href=\"https://res.layui.com/layui/dist/css/layui.css?t=1545041465480\" media=\"all\">"
        #头部
        header = "<!DOCTYPE HTML><html lang=\"en-US\"><head><meta charset=\"UTF-8\"><title>爬虫抓取豆瓣电影实例</title>"+link+"</head><body><table class=\"layui-table\">"
        #尾部
        footer = "</table></body></html>"
        
        Ttitle = "<tr><td><b>序号</b></td><td><b>电影名</b></td><td><b>电影别名</b></td><td><b>观看链接</b></td></tr>" 
        append = ""
        table = "<tr><td>{}</td><td>{}</td><td>{}</td><td><a href=\"{}\">点击观看</a></td></tr>"
        file = open(filename,"w",encoding = "utf-8")
        file.write(header)
        for i in range(len(ulist)):
          td1 = ulist[i][0]
          td2 = ulist[i][1]
          url = ulist[i][2]
          append += table.format(i+1,td1,td2,url)
        file.write(Ttitle)
        file.write(append)
        file.write(footer)
        file.close()
        #print(append)

#其他文件输出格式
def fileOut(ulist,filename):
    temp = filename.split(".")[1]#检查是否是html格式如果是则Flag=True
    if(temp == "html"):
        wirterFile(ulist,filename,True)
    else:
        append = ""
        file = open(filename,"w",encoding = "utf-8")
        tplt = "{0:^10}\t{1:{4}^10}\t{2:<20}\t{3:>50}\n"
        file.write(tplt.format("序号","电影名","电影别名","观看链接",chr(12288)))
        file.write(tplt.format("-----","-----","-------","-------","-"))
        for i in range(len(ulist)):
          td1 = ulist[i][0]
          td2 = ulist[i][1]
          url = ulist[i][2]
          append += tplt.format(i+1,td1,td2,url,chr(12288))
        file.write(append)
        file.write(tplt.format("-----","-----","-------","-------","-"))
        file.close()

        
def main():
    #数据列表
    ulist = []
    #显示条数....若未加载全部数据只能设置num<=20
    num = 20 
    #分页开始页条数：0 20 40...根据抓取网站设置仅支持20*N
    startPage = "0"
    url = "https://movie.douban.com/tag/top100?start="+startPage+"&type=S"
    html = getHTMLText(url)
    #数据处理
    fillVidoList(ulist,html)
    #打印到屏幕
    output(ulist,num)
    #写入文件默认为html格式Flag=True,自定义格式Flag=False
    #输出文件名  修改请求页面时文件自动改变
    filename = "view"+startPage+".html"
    Flag = True
    wirterFile(ulist,filename,Flag)

main()
